#!/usr/bin/perl -w
use strict;
use warnings;
use DBI;
use LWP::Protocol::https;
use LWP::UserAgent;
use Digest::MD5 qw(md5 md5_hex md5_base64);
use Encode;
use utf8;
binmode(STDIN,':encoding(utf8)');
binmode(STDOUT,':encoding(utf8)');
binmode(STDERR,':encoding(utf8)'); 
use Time::Local;
use Date::Parse;

&insert_news();

sub insert_news{
    my $filevar2;
	my $flag=0;
    my $count;
	my $driver="DBI:mysql";
	my $database="news";
	my $host="localhost";
	my $user="root";
	my $password="192510";
	my $db=DBI->connect("$driver:database=$database;host=$host;user=$user;password=$password");
	 
	my $url="https://raw.githubusercontent.com/offensive-security/exploit-database/master/files.csv";
	my ($ua, $request , $response , $content);
	$ua = LWP::UserAgent->new;
	$request = HTTP::Request->new('GET',$url);
	$response = $ua->request($request);
	$content = $response->content;
	my $html=decode("utf-8",$content);
	
	my @split_n=split(/\n/,$html);
	my $split_n_num = scalar @split_n;
	print $split_n_num;
	for(my $i=1;$i<$split_n_num;$i++){
	   
		 my $arr_qun=$split_n[$i];
		 my @split_dh=split(/,/,$arr_qun);
		 
		 my $id=$split_dh[0];
		 my $file=$split_dh[1];#得到文件
		  
		 my $title=$split_dh[2];#得到标题
		 
		 if($title=~m/"/){
		    $title=~s/"//g;
		 }
		 
		 my $data=$split_dh[3];#得到日期
		 my $author=$split_dh[4];#得到作者
		 	 
		 if($author=~m/"/){
		    $author=~s/"//g;
		 }
		 
		 #内容存的是文件的名字(显示的时候可以直接打开文件)
		 my @split_xg=split(/\//,$file);
		 my $split_xg_num=scalar @split_xg;
		 my $file_name=$split_xg[$split_xg_num-1];#$file_name当内容存入数据库中
		
		 #md5
		 my $m=$id.$title;
		 Encode::_utf8_off($m);
		 my $md5=md5($m);
		 
		 #web
		 my $web=6;
		 
		 #link
		 my $link="https://raw.githubusercontent.com/offensive-security/exploit-database/master/$file";
		 my ($ua1, $request1 , $response1 , $content1);
		 
		 if($i%100==0){
			sleep(30);
		 }
		 $ua1 = LWP::UserAgent->new;
		 #随机模拟浏览器
	     my $agentstr=&randomagent;
         $ua1->agent('User-Agent'=>$agentstr);
		 
		 $request1 = HTTP::Request->new('GET',$link);
		 $response1 = $ua1->request($request1);
		 
		 if($response1->code ne '200'){
		 
		 }else{
		    $content1 = $response1->content;
		    my $html2=decode("utf-8",$content1);#得到文件的内容
		 
			my $bj_publishtime=str2time("$data GMT");
			$title=&fanyi($title);
			#开始入库
			#先去掉变量的空格在判断
			my $q_name=$file;
			$q_name=~s/\s//g;
				 
			my $q_rel_href=$link;
			$q_rel_href=~s/\s//g;
				 
			my $q_rel_author=$author;
			$q_rel_author=~s/\s//g;
			
			
			 if($q_name ne"" and $q_rel_href ne"" and $q_rel_author ne""){
				   
					 my $sth = $db->prepare(q{insert ignore into news(title,web,author,publishtime,content,link,nmd5)VALUES(?,?,?,?,?,?,?)}) or die $db->errstr;
					 $sth->execute($title,$web,$author,$data,$file_name,$link,$md5) or die $db->errstr;
					
					 #数据中插入,在把文件下载到wenjian目录下
					 my $wenjian="/var/www/html/wangzhan/wenjian/$file_name";
					
					 if(-e $wenjian){
					
					 }else{
					
					  my $cj_e=`cd /var/www/html/wangzhan/wenjian/ && touch $file_name`;
					  my $qunxian=`cd /var/www/html/wangzhan/wenjian/ && chmod 777 $file_name`;
					  open($filevar2,">/var/www/html/wangzhan/wenjian/$file_name")|| die("Could not open file");
					  #把内容写入到新建的file_name文件中
					  print $filevar2 $html2;
					  close($filevar2);
					
					 }
					
					
					
			 }
						
		 
		 }
		 
	
	}
	
}

sub fanyi(){
    my $fanyi= $_[0];
    #把标题进行翻译
	if($fanyi=~/Overflow/){
		$fanyi=~s/Overflow/内存溢出/g;
	}
	
	if($fanyi=~/Command/){
		$fanyi=~s/Command/命令/g;
	}
	
	if($fanyi=~/Buffer/){
		$fanyi=~s/Buffer/缓冲区/g;
	}
	
	if($fanyi=~/Remote/){
		$fanyi=~s/Remote/远程/g;
	}
	
    return $fanyi;
}


#模拟随机浏览器
sub randomagent
{
## FireFox
# Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0
# Mozilla/5.0 (Windows NT 6.1; WOW64; rv:13.0) Gecko/20100101 Firefox/13.0.1
# Mozilla/5.0 (Windows NT 6.1; WOW64; rv:14.0) Gecko/20100101 Firefox/14.0.1
# Chrome
# Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.168 Safari/535.19 QIHU 360EE
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1
##Safari
# Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13
##Sogou
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0
##Baidu
# Mozilla/5.0 (Windows; U; Windows NT 6.1; zh_CN) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/18.0 BIDUBrowser/2.6 Safari/534.7
##QQ
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.7 (KHTML, like Gecko) Chrome/20.0.1099.0 Safari/536.7 QQBrowser/6.14.15138.201
##Maxthon
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.12 (KHTML, like Gecko) Maxthon/3.4.2.3000 Chrome/18.0.966.0 Safari/535.12
##LBBROWSER
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER
##TheWorld
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.79 Safari/535.11 QIHU THEWORLD
	my @agent_conf=('Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0',
	'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:13.0) Gecko/20100101 Firefox/13.0.1',
	'Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.168 Safari/535.19 QIHU 360EE',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1',
	'Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
	'Mozilla/5.0 (Windows; U; Windows NT 6.1; zh_CN) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/18.0 BIDUBrowser/2.6 Safari/534.7',
	'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.12 (KHTML, like Gecko) Maxthon/3.4.2.3000 Chrome/18.0.966.0 Safari/535.12',
	'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER',
	'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.79 Safari/535.11 QIHU THEWORLD');
	my $int=int(rand(12));
	return $agent_conf[$int];
}
